{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa460049-640b-4961-981a-affc463c8a3b",
   "metadata": {},
   "source": [
    "# 1 准备工作"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3956f857-03ce-481c-bb42-abbe139311bd",
   "metadata": {},
   "source": [
    "## 1.1 安装相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3fb17e0f-b216-45a1-874a-89b4aedbd88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !apt update\n",
    "# !apt install unrar -y\n",
    "# !apt install mdbtools -y\n",
    "# !apt install default-jre -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "acee0a2a-18b5-463a-adb4-d636acf3ea62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !java --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bb4535f7-9daf-474e-86a0-301054e782c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pytz\n",
    "# !pip install rarfile\n",
    "# !pip install jaydebeapi\n",
    "# !pip install pyodbc\n",
    "# !pip install uuid\n",
    "# !pip install tenacity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78abd657-675f-4d8c-8251-52752fef53b5",
   "metadata": {},
   "source": [
    "## 1.2 加载包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a0df1bd-91b1-476d-bf6a-9c17983db382",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pytz\n",
    "from datetime import datetime\n",
    "import json, re, gc\n",
    "import glob\n",
    "import os, sys, rarfile\n",
    "from tqdm import tqdm\n",
    "# from typing import Dict, Any\n",
    "from pathlib import Path\n",
    "\n",
    "import platform\n",
    "import jaydebeapi\n",
    "import jpype\n",
    "import jpype.imports\n",
    "from jpype.types import *\n",
    "import subprocess\n",
    "import pyodbc, io\n",
    "import difflib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import torch\n",
    "import pickle\n",
    "# from openai import OpenAI\n",
    "import warnings, logging\n",
    "sys.path.append('./')\n",
    "\n",
    "from utils.load_access_data import *\n",
    "from utils.file_unrar import *\n",
    "# from utils.data_mapping import *\n",
    "# from utils.data_mapping_36country3 import *\n",
    "# from utils.invoke_llm import *\n",
    "from utils.file_process import *\n",
    "from utils.bash_mapping import *\n",
    "from utils.logging_config import setup_logger\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "logger = setup_logger('Data Goverance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d22cb7f-a935-4e82-bb54-a1cc22e6d6a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1e70e112-297d-41ad-b94f-ef1c9bb884e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-17 07:05:13,601 - Data Goverance - INFO - 2025-09-17 15:05:13.600928+08:00\n"
     ]
    }
   ],
   "source": [
    "# 计时\n",
    "tz_beijing = pytz.timezone('Asia/Shanghai')\n",
    "bj_now = datetime.now(tz_beijing)\n",
    "dt_now = datetime.strptime(datetime.strftime(bj_now, '%Y-%m-%d'), '%Y-%m-%d')\n",
    "logger.info(bj_now)\n",
    "\n",
    "all_start=time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14176eb0-f67a-4053-96f8-ff299df631ae",
   "metadata": {},
   "source": [
    "## 1.3 准备全局变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2baa7fe2-b85d-4299-b235-d672e338ab78",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config:\n",
    "    target_cols = ['channelno', 'dataid', 'iesign', 'datatype', 'writeoffflag'\n",
    "            , 'writeoffdataid', 'outputdate', 'origincountrycode', 'origincountry', 'countrycodeofdelivery'\n",
    "            , 'countryofdelivery', 'importername', 'importeraddress', 'importercontact', 'suppliername'\n",
    "            , 'supplieraddress', 'suppliercontact', 'hscode', 'hscodedescription', 'commoditydescription'\n",
    "            , 'totalcifvalue', 'totalfobvalue', 'grossweight', 'netweight', 'quantity'\n",
    "            , 'quantityunit', 'teu', 'importer_forwarderagent', 'supplier_forwarderagent', 'abnormaldata'\n",
    "            , 'portofloading', 'portofdestination', 'loadingcountrycode', 'loadingcountry', 'transportterm'\n",
    "            , 'tradeterm', 'paymentterm', 'carrier', 'containerno', 'vesselname'\n",
    "            , 'brand', 'version', 'country', 'IMPORTER_ID', 'SUPPLIER_ID', 'cif_currency', 'fob_currency']\n",
    "config = Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "90881866-9cbb-4442-a5f6-7217e901ce3a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 加载国家映射表\n",
    "country_code = pd.read_csv('param_files/国家代码表.csv')\n",
    "country_code_dict = country_code.loc[:,['英文国名','中文国名']].to_dict('records')\n",
    "country_code_dict_detail = country_code.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67fbe1a5-8a7b-4d95-ae58-ef5a97913cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载36个国家码\n",
    "data_country36_code = pd.read_csv('param_files/数据36国家代码表.csv')\n",
    "data_country36_dict = data_country36_code.to_dict('records')\n",
    "\n",
    "country36_dict = dict()\n",
    "for x in data_country36_dict:\n",
    "    country36_dict[x.get('中文国名')]={'英文国名': x.get('英文国名'),'国别编码':x.get('国别编码')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ff0099c1-7091-40f2-8fe0-00618ab32c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理的中间结果，\n",
    "model_result = pickle.load(open('param_files/model_rslt_05_20250810.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0158c76f-e0df-44d3-9235-12945187e89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "country36_cols_dict = model_result['all_country_cols_dict']\n",
    "all_country_cols_dict = model_result['all_country_cols_dict']\n",
    "\n",
    "all_country_list = model_result['all_country_list']\n",
    "llm_country_describe = model_result['llm_country_describe']\n",
    "llm_country_mapping = model_result['llm_country_mapping']\n",
    "country_mapping = model_result['country_mapping']\n",
    "currency_df = model_result['currency_df']\n",
    "currency_on_usd = model_result['currency_on_usd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "919d9b79-5ccd-4bc7-a1f6-704526b90632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['all_country_cols_dict', 'all_country_list', 'llm_country_describe', 'llm_country_mapping', 'country_mapping', 'currency_df', 'currency_on_usd'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_result.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eb6ed96d-7eef-45e7-acbb-4ffeeb3760ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(model_result['country_mapping'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e5294ce-9f93-48ed-85f7-197a6c1ab582",
   "metadata": {},
   "source": [
    "# 2 空值率与指标统计，并更新数字"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26f913de-8799-42ed-b6fb-178a6a5263ba",
   "metadata": {},
   "source": [
    "## 2.1 总和指标计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cea3c8e7-839f-4afc-a348-e75775489517",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['坦桑尼亚']\n",
    "country_list = ['乌克兰']\n",
    "csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/files11')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "322d792b-eb24-47c3-99e0-ce9a24602ed1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-17 07:06:24,252 - File Process - INFO - 乌克兰\n",
      "2025-09-17 07:06:24,254 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_EXP.csv\n",
      "2025-09-17 07:06:24,889 - File Process - INFO - tab_tmp shape: (133312, 45)\n",
      "2025-09-17 07:06:25,407 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:25,409 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_IMP.csv\n",
      "2025-09-17 07:06:29,014 - File Process - INFO - tab_tmp shape: (755728, 45)\n",
      "2025-09-17 07:06:31,992 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:31,993 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv\n",
      "2025-09-17 07:06:32,648 - File Process - INFO - tab_tmp shape: (130408, 45)\n",
      "2025-09-17 07:06:33,202 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:33,202 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_EXP.csv\n",
      "2025-09-17 07:06:33,886 - File Process - INFO - tab_tmp shape: (146180, 45)\n",
      "2025-09-17 07:06:34,446 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:34,447 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_EXP.csv\n",
      "2025-09-17 07:06:35,086 - File Process - INFO - tab_tmp shape: (134643, 45)\n",
      "2025-09-17 07:06:35,614 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:35,614 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_IMP.csv\n",
      "2025-09-17 07:06:38,939 - File Process - INFO - tab_tmp shape: (652325, 45)\n",
      "2025-09-17 07:06:41,306 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:41,306 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_IMP.csv\n",
      "2025-09-17 07:06:44,663 - File Process - INFO - tab_tmp shape: (771556, 45)\n",
      "2025-09-17 07:06:47,519 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:47,520 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_EXP.csv\n",
      "2025-09-17 07:06:48,363 - File Process - INFO - tab_tmp shape: (114237, 45)\n",
      "2025-09-17 07:06:48,872 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:48,873 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_IMP.csv\n",
      "2025-09-17 07:06:52,046 - File Process - INFO - tab_tmp shape: (744467, 45)\n",
      "2025-09-17 07:06:54,748 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:06:54,749 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_IMP.csv\n",
      "2025-09-17 07:06:58,050 - File Process - INFO - tab_tmp shape: (739131, 45)\n",
      "2025-09-17 07:07:00,799 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:07:00,800 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_IMP.csv\n",
      "2025-09-17 07:07:03,864 - File Process - INFO - tab_tmp shape: (675815, 45)\n",
      "2025-09-17 07:07:06,381 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:07:06,382 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_EXP.csv\n",
      "2025-09-17 07:07:07,069 - File Process - INFO - tab_tmp shape: (140254, 45)\n",
      "2025-09-17 07:07:07,647 - File Process - INFO - stat_tmp shape: (1, 9)\n",
      "2025-09-17 07:07:07,681 - Data Goverance - INFO - (12, 9)\n",
      "2025-09-17 07:07:07,688 - Data Goverance - INFO - get_type_enumvalue takes 0.7238863388697306 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], csv_files_path, country36_dict) for ctry in country_list]\n",
    "results = []\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: compute_stat4(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        res = future.result()\n",
    "        logger.info(res.shape)\n",
    "        results.append(res)\n",
    "\n",
    "stat_rslt1 = pd.concat([x for x in results], axis=0, ignore_index=True)\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "13d2681a-6c8b-4c90-a2c2-af0d9e3f4344",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 单线程\n",
    "# stat_rslt1 = compute_stat4(country_list, csv_files_path, country36_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "69fe5b33-ff94-4ab8-89f7-e1f0d8eba209",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stat_rslt1.to_csv('result/statistic_result_sum_20250820.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# stat_rslt1.to_csv('result/statistic_result_sum_20250908.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "stat_rslt1.to_csv('result/statistic_result_sum_20250917.csv', sep=',', index=False, header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9b9fb37b-188d-4d9e-b104-7fe6c7924830",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>进出口</th>\n",
       "      <th>数据量</th>\n",
       "      <th>cif总价</th>\n",
       "      <th>fob总价</th>\n",
       "      <th>毛重</th>\n",
       "      <th>重量</th>\n",
       "      <th>数量</th>\n",
       "      <th>国家</th>\n",
       "      <th>文件路径</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E</td>\n",
       "      <td>133312</td>\n",
       "      <td>3.711151e+09</td>\n",
       "      <td>4.977025e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.165310e+10</td>\n",
       "      <td>1.165310e+10</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I</td>\n",
       "      <td>755728</td>\n",
       "      <td>1.342772e+10</td>\n",
       "      <td>6.370281e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.057001e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E</td>\n",
       "      <td>130408</td>\n",
       "      <td>5.397415e+09</td>\n",
       "      <td>4.379545e+09</td>\n",
       "      <td>1.715228e+08</td>\n",
       "      <td>1.584243e+10</td>\n",
       "      <td>1.567183e+10</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>E</td>\n",
       "      <td>146180</td>\n",
       "      <td>4.535919e+09</td>\n",
       "      <td>6.318823e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.482009e+10</td>\n",
       "      <td>1.482009e+10</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>E</td>\n",
       "      <td>134643</td>\n",
       "      <td>3.321749e+09</td>\n",
       "      <td>3.332823e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.313279e+09</td>\n",
       "      <td>8.313279e+09</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>I</td>\n",
       "      <td>652325</td>\n",
       "      <td>6.136305e+09</td>\n",
       "      <td>5.773970e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.195145e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I</td>\n",
       "      <td>771556</td>\n",
       "      <td>6.516675e+09</td>\n",
       "      <td>1.018680e+10</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.050862e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>E</td>\n",
       "      <td>114237</td>\n",
       "      <td>6.037187e+09</td>\n",
       "      <td>6.036727e+09</td>\n",
       "      <td>1.645595e+10</td>\n",
       "      <td>1.641248e+10</td>\n",
       "      <td>2.098773e+08</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>I</td>\n",
       "      <td>744467</td>\n",
       "      <td>1.044280e+10</td>\n",
       "      <td>6.410422e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.703817e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>I</td>\n",
       "      <td>739131</td>\n",
       "      <td>6.495239e+09</td>\n",
       "      <td>5.895575e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.718354e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I</td>\n",
       "      <td>675815</td>\n",
       "      <td>5.370704e+09</td>\n",
       "      <td>8.938194e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.283560e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>E</td>\n",
       "      <td>140254</td>\n",
       "      <td>5.792442e+09</td>\n",
       "      <td>4.739790e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.660882e+10</td>\n",
       "      <td>1.660882e+10</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   进出口     数据量         cif总价         fob总价            毛重            重量  \\\n",
       "0    E  133312  3.711151e+09  4.977025e+09  0.000000e+00  1.165310e+10   \n",
       "1    I  755728  1.342772e+10  6.370281e+09  0.000000e+00  4.057001e+09   \n",
       "2    E  130408  5.397415e+09  4.379545e+09  1.715228e+08  1.584243e+10   \n",
       "3    E  146180  4.535919e+09  6.318823e+09  0.000000e+00  1.482009e+10   \n",
       "4    E  134643  3.321749e+09  3.332823e+09  0.000000e+00  8.313279e+09   \n",
       "5    I  652325  6.136305e+09  5.773970e+09  0.000000e+00  3.195145e+09   \n",
       "6    I  771556  6.516675e+09  1.018680e+10  0.000000e+00  4.050862e+09   \n",
       "7    E  114237  6.037187e+09  6.036727e+09  1.645595e+10  1.641248e+10   \n",
       "8    I  744467  1.044280e+10  6.410422e+09  0.000000e+00  3.703817e+09   \n",
       "9    I  739131  6.495239e+09  5.895575e+09  0.000000e+00  3.718354e+09   \n",
       "10   I  675815  5.370704e+09  8.938194e+09  0.000000e+00  3.283560e+09   \n",
       "11   E  140254  5.792442e+09  4.739790e+09  0.000000e+00  1.660882e+10   \n",
       "\n",
       "              数量   国家                                               文件路径  \n",
       "0   1.165310e+10  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "1   0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "2   1.567183e+10  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "3   1.482009e+10  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "4   8.313279e+09  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "5   0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "6   0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "7   2.098773e+08  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "8   0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "9   0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "10  0.000000e+00  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "11  1.660882e+10  乌克兰  dataset/data_20250724/20251t3_csv_final_tmp/fi...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_rslt1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "761d9e88-ff0f-4c68-9256-e5dc0ba8db00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# future.result().shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c699184-2580-4723-81b7-b380d2fb8460",
   "metadata": {},
   "source": [
    "## 2.2 空值率指标计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "15b35f4a-f893-43b7-82a2-4799092cf839",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['坦桑尼亚']\n",
    "country_list = ['乌克兰']\n",
    "# csv_files_path = Path('dataset/data_20250724/20251t3_csv_final/files')\n",
    "# csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/final_files')\n",
    "csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/files11')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ad3ecb31-8190-49e5-9c60-865eda31a9c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-17 07:07:07,722 - File Process - INFO - 乌克兰\n",
      "2025-09-17 07:07:07,724 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_EXP.csv\n",
      "2025-09-17 07:07:08,356 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_EXP.csv, the shape is (133312, 45)\n",
      "2025-09-17 07:07:09,629 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_IMP.csv\n",
      "2025-09-17 07:07:13,117 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_IMP.csv, the shape is (755728, 45)\n",
      "2025-09-17 07:07:20,043 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv\n",
      "2025-09-17 07:07:20,752 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv, the shape is (130408, 45)\n",
      "2025-09-17 07:07:21,971 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_EXP.csv\n",
      "2025-09-17 07:07:22,644 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_EXP.csv, the shape is (146180, 45)\n",
      "2025-09-17 07:07:24,025 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_EXP.csv\n",
      "2025-09-17 07:07:24,648 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_EXP.csv, the shape is (134643, 45)\n",
      "2025-09-17 07:07:25,915 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_IMP.csv\n",
      "2025-09-17 07:07:29,252 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_IMP.csv, the shape is (652325, 45)\n",
      "2025-09-17 07:07:35,260 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_IMP.csv\n",
      "2025-09-17 07:07:38,643 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_IMP.csv, the shape is (771556, 45)\n",
      "2025-09-17 07:07:45,714 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_EXP.csv\n",
      "2025-09-17 07:07:46,573 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_EXP.csv, the shape is (114237, 45)\n",
      "2025-09-17 07:07:47,968 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_IMP.csv\n",
      "2025-09-17 07:07:51,146 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_IMP.csv, the shape is (744467, 45)\n",
      "2025-09-17 07:07:58,047 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_IMP.csv\n",
      "2025-09-17 07:08:01,475 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_IMP.csv, the shape is (739131, 45)\n",
      "2025-09-17 07:08:08,297 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_IMP.csv\n",
      "2025-09-17 07:08:11,503 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_IMP.csv, the shape is (675815, 45)\n",
      "2025-09-17 07:08:17,730 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_EXP.csv\n",
      "2025-09-17 07:08:18,446 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_EXP.csv, the shape is (140254, 45)\n"
     ]
    }
   ],
   "source": [
    "stat_rslt2 = compute_stat5(country_list, csv_files_path, country36_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dee180d9-2862-4db4-b03f-1b129bee5632",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-17 07:08:19,782 - File Process - INFO - 乌克兰\n",
      "2025-09-17 07:08:19,783 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_EXP.csv\n",
      "2025-09-17 07:08:20,418 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_EXP.csv, the shape is (133312, 45)\n",
      "2025-09-17 07:08:21,680 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_IMP.csv\n",
      "2025-09-17 07:08:25,110 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_IMP.csv, the shape is (755728, 45)\n",
      "2025-09-17 07:08:32,116 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv\n",
      "2025-09-17 07:08:32,813 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv, the shape is (130408, 45)\n",
      "2025-09-17 07:08:34,062 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_EXP.csv\n",
      "2025-09-17 07:08:34,763 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202505_EXP.csv, the shape is (146180, 45)\n",
      "2025-09-17 07:08:36,178 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_EXP.csv\n",
      "2025-09-17 07:08:36,825 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_EXP.csv, the shape is (134643, 45)\n",
      "2025-09-17 07:08:38,124 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_IMP.csv\n",
      "2025-09-17 07:08:41,443 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_IMP.csv, the shape is (652325, 45)\n",
      "2025-09-17 07:08:47,366 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_IMP.csv\n",
      "2025-09-17 07:08:50,785 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_IMP.csv, the shape is (771556, 45)\n",
      "2025-09-17 07:08:57,759 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_EXP.csv\n",
      "2025-09-17 07:08:58,632 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202501_EXP.csv, the shape is (114237, 45)\n",
      "2025-09-17 07:09:00,000 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_IMP.csv\n",
      "2025-09-17 07:09:03,182 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202506_IMP.csv, the shape is (744467, 45)\n",
      "2025-09-17 07:09:09,935 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_IMP.csv\n",
      "2025-09-17 07:09:13,335 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202504_IMP.csv, the shape is (739131, 45)\n",
      "2025-09-17 07:09:20,020 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_IMP.csv\n",
      "2025-09-17 07:09:23,194 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_IMP.csv, the shape is (675815, 45)\n",
      "2025-09-17 07:09:29,298 - File Process - INFO - dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_EXP.csv\n",
      "2025-09-17 07:09:29,991 - File Process - INFO - the file dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202503_EXP.csv, the shape is (140254, 45)\n",
      "2025-09-17 07:09:31,300 - Data Goverance - INFO - (12, 35)\n",
      "2025-09-17 07:09:31,302 - Data Goverance - INFO - get_type_enumvalue takes 1.192027747631073 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "params_list = [([ctry], csv_files_path, country36_dict) for ctry in country_list]\n",
    "\n",
    "results = []\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: compute_stat5(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        res = future.result()\n",
    "        logger.info(res.shape)\n",
    "        results.append(res)\n",
    "\n",
    "stat_rslt2 = pd.concat([x for x in results], axis=0, ignore_index=True)\n",
    "# stat_rslt2\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3549e3f8-0421-4ff9-a1dd-0f78589e87be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stat_rslt2.to_csv('result/statistic_result_rus_20250820.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# stat_rslt2.to_csv('result/statistic_result_rus_20250908.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "stat_rslt2.to_csv('result/statistic_result_rus_20250917.csv', sep=',', index=False, header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "384e3d14-9211-41f7-a23c-1024c00e8792",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>进出口</th>\n",
       "      <th>数据量</th>\n",
       "      <th>目的国国际编码</th>\n",
       "      <th>原产国国际编码</th>\n",
       "      <th>启运国国际编码</th>\n",
       "      <th>hs编码</th>\n",
       "      <th>hs编码描述</th>\n",
       "      <th>产品描述</th>\n",
       "      <th>cif总价</th>\n",
       "      <th>fob总价</th>\n",
       "      <th>...</th>\n",
       "      <th>付款方式</th>\n",
       "      <th>启运港</th>\n",
       "      <th>目的港</th>\n",
       "      <th>集装箱箱号</th>\n",
       "      <th>启运国英文名称</th>\n",
       "      <th>承运人名称</th>\n",
       "      <th>船名</th>\n",
       "      <th>品牌</th>\n",
       "      <th>国家</th>\n",
       "      <th>文件路径</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E</td>\n",
       "      <td>133312</td>\n",
       "      <td>133312</td>\n",
       "      <td>133312</td>\n",
       "      <td>133312</td>\n",
       "      <td>133312</td>\n",
       "      <td>0</td>\n",
       "      <td>133312</td>\n",
       "      <td>133312</td>\n",
       "      <td>133311</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>133312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I</td>\n",
       "      <td>755728</td>\n",
       "      <td>755728</td>\n",
       "      <td>755728</td>\n",
       "      <td>736963</td>\n",
       "      <td>755728</td>\n",
       "      <td>0</td>\n",
       "      <td>755728</td>\n",
       "      <td>755728</td>\n",
       "      <td>755728</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>755659</td>\n",
       "      <td>0</td>\n",
       "      <td>736963</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E</td>\n",
       "      <td>130408</td>\n",
       "      <td>130408</td>\n",
       "      <td>130408</td>\n",
       "      <td>130408</td>\n",
       "      <td>130408</td>\n",
       "      <td>0</td>\n",
       "      <td>130407</td>\n",
       "      <td>130404</td>\n",
       "      <td>130408</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>271</td>\n",
       "      <td>130408</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>E</td>\n",
       "      <td>146180</td>\n",
       "      <td>146180</td>\n",
       "      <td>146180</td>\n",
       "      <td>146180</td>\n",
       "      <td>146180</td>\n",
       "      <td>0</td>\n",
       "      <td>146180</td>\n",
       "      <td>146180</td>\n",
       "      <td>146165</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>146180</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>E</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>0</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>134643</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>134643</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>I</td>\n",
       "      <td>652325</td>\n",
       "      <td>652325</td>\n",
       "      <td>652325</td>\n",
       "      <td>652325</td>\n",
       "      <td>652325</td>\n",
       "      <td>0</td>\n",
       "      <td>652177</td>\n",
       "      <td>652325</td>\n",
       "      <td>652325</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>636900</td>\n",
       "      <td>0</td>\n",
       "      <td>652325</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I</td>\n",
       "      <td>771556</td>\n",
       "      <td>771556</td>\n",
       "      <td>771556</td>\n",
       "      <td>755332</td>\n",
       "      <td>771556</td>\n",
       "      <td>0</td>\n",
       "      <td>771555</td>\n",
       "      <td>771556</td>\n",
       "      <td>771556</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>755332</td>\n",
       "      <td>0</td>\n",
       "      <td>755332</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>E</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>0</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>114237</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12914</td>\n",
       "      <td>114237</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>I</td>\n",
       "      <td>744467</td>\n",
       "      <td>744467</td>\n",
       "      <td>744467</td>\n",
       "      <td>744403</td>\n",
       "      <td>744467</td>\n",
       "      <td>0</td>\n",
       "      <td>744464</td>\n",
       "      <td>744467</td>\n",
       "      <td>744467</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>744403</td>\n",
       "      <td>0</td>\n",
       "      <td>744403</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>I</td>\n",
       "      <td>739131</td>\n",
       "      <td>739131</td>\n",
       "      <td>739131</td>\n",
       "      <td>722960</td>\n",
       "      <td>739131</td>\n",
       "      <td>0</td>\n",
       "      <td>739130</td>\n",
       "      <td>739131</td>\n",
       "      <td>739131</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>722960</td>\n",
       "      <td>0</td>\n",
       "      <td>722960</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I</td>\n",
       "      <td>675815</td>\n",
       "      <td>675815</td>\n",
       "      <td>675815</td>\n",
       "      <td>660489</td>\n",
       "      <td>675815</td>\n",
       "      <td>0</td>\n",
       "      <td>675815</td>\n",
       "      <td>675815</td>\n",
       "      <td>675815</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>660489</td>\n",
       "      <td>0</td>\n",
       "      <td>660489</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>E</td>\n",
       "      <td>140254</td>\n",
       "      <td>140254</td>\n",
       "      <td>140254</td>\n",
       "      <td>140254</td>\n",
       "      <td>140254</td>\n",
       "      <td>0</td>\n",
       "      <td>140254</td>\n",
       "      <td>140251</td>\n",
       "      <td>140254</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140254</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>乌克兰</td>\n",
       "      <td>dataset/data_20250724/20251t3_csv_final_tmp/fi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   进出口     数据量 目的国国际编码 原产国国际编码 启运国国际编码    hs编码 hs编码描述    产品描述   cif总价   fob总价  \\\n",
       "0    E  133312  133312  133312  133312  133312      0  133312  133312  133311   \n",
       "1    I  755728  755728  755728  736963  755728      0  755728  755728  755728   \n",
       "2    E  130408  130408  130408  130408  130408      0  130407  130404  130408   \n",
       "3    E  146180  146180  146180  146180  146180      0  146180  146180  146165   \n",
       "4    E  134643  134643  134643  134643  134643      0  134643  134643  134643   \n",
       "5    I  652325  652325  652325  652325  652325      0  652177  652325  652325   \n",
       "6    I  771556  771556  771556  755332  771556      0  771555  771556  771556   \n",
       "7    E  114237  114237  114237  114237  114237      0  114237  114237  114237   \n",
       "8    I  744467  744467  744467  744403  744467      0  744464  744467  744467   \n",
       "9    I  739131  739131  739131  722960  739131      0  739130  739131  739131   \n",
       "10   I  675815  675815  675815  660489  675815      0  675815  675815  675815   \n",
       "11   E  140254  140254  140254  140254  140254      0  140254  140251  140254   \n",
       "\n",
       "    ... 付款方式 启运港     目的港  集装箱箱号 启运国英文名称 承运人名称 船名 品牌   国家  \\\n",
       "0   ...    0   0       0      0  133312     0  0  0  乌克兰   \n",
       "1   ...    0   0  755659      0  736963     0  0  0  乌克兰   \n",
       "2   ...    0   0       0    271  130408     0  0  0  乌克兰   \n",
       "3   ...    0   0       0      0  146180     0  0  0  乌克兰   \n",
       "4   ...    0   0       0      0  134643     0  0  0  乌克兰   \n",
       "5   ...    0   0  636900      0  652325     0  0  0  乌克兰   \n",
       "6   ...    0   0  755332      0  755332     0  0  0  乌克兰   \n",
       "7   ...    0   0       0  12914  114237     0  0  0  乌克兰   \n",
       "8   ...    0   0  744403      0  744403     0  0  0  乌克兰   \n",
       "9   ...    0   0  722960      0  722960     0  0  0  乌克兰   \n",
       "10  ...    0   0  660489      0  660489     0  0  0  乌克兰   \n",
       "11  ...    0   0       0      0  140254     0  0  0  乌克兰   \n",
       "\n",
       "                                                 文件路径  \n",
       "0   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "1   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "2   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "3   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "4   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "5   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "6   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "7   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "8   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "9   dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "10  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "11  dataset/data_20250724/20251t3_csv_final_tmp/fi...  \n",
       "\n",
       "[12 rows x 35 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_rslt2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ca44b1d4-dd32-4216-8b92-59e093c9bdea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wc -l dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52802231-a7f5-4877-9278-199939970fd9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d342a52c-d4e7-491b-bbd7-82ebc1df4d32",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d598c13-4a73-4ce4-8e15-a56aeca99cd9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "25a80fff-3bd3-44f0-be12-b36efa246b6a",
   "metadata": {},
   "source": [
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "893235b3-693f-47df-bad1-6256b394e865",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 07:06:32,450 - Data Goverance - INFO - 18.75146779616674 minutes\n",
      "2025-09-10 07:06:32,451 - Data Goverance - INFO - All done!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_end=time.time()\n",
    "logger.info(f'{(all_end-all_start)/60} minutes')\n",
    "logger.info('All done!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "409ea3bf-dca2-4bd1-b59d-ef2045e32deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# str_type = dict()\n",
    "# for col in config.target_cols:\n",
    "#     str_type[col] = 'str'\n",
    "# str_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c309bd2d-0ddb-4aeb-a788-672b2daea563",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "763f4c3c-4b7e-4f67-85ba-450a11443419",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
